from v4.Spider import Spider
from v4.SpiderModule import SpiderModule
from v4.test.TestPipeline import TestPipeline
from v4.test.TestPageProcessor import TestProcessor
from v4.SockPuppet import SockPuppet
from v4.RequestItem import RequestItem
from v4.test.headers_list import headers_list


if __name__ == '__main__':
    sock_puppet = SockPuppet()
    spider_module = SpiderModule(sock_puppet=sock_puppet,
                                 pipeline_class=TestPipeline,
                                 page_processor_class=TestProcessor)
    request_items = []
    for i in range(61, 50, -1):
        request_items.append(RequestItem(url='https://m.weibo.cn/api/comments/show',
                                         params={'id': '4620209346582917', 'page': str(i)}, timeout=2))

    spider = Spider(spider_module_list=[spider_module], thread_num=3, request_items=request_items)
    spider.run()